# Updated by Wei 20141007 night at school
# Updated by Wei 20140329 afternoon at school
# build the denominator of the relRank
print "Program Begins..."
# function listings:
# function unknown
# stepMinus1()
# function unknown
# step0()
# actual build of the denominator table
# step1()

def stepMinus1():
    # the threshold being 50
    # original step factor
    # stepFactor = float(1.0 / 2.0)
    # modified step factor
    stepFactor = float(1.0 / 3.0)
    # test
    # print int(100.0 * stepFactor)
    
    outputFileName = "/home/diaosi/gov2ClearYourMindAndDoItAgain/relRankFilesRelated/predefinedClassLabelLowerBoundBasedOnListLength_numOfPieces_stepGap_2_20140118Night"
    outputFileHandler = open(outputFileName,"w")
    
    inputFileName = "/home/diaosi/gov2ClearYourMindAndDoItAgain/relrankWithLengthAnalyze20140102Night/version2_20140112Night/predefinedClassLabelLowerBoundOfListLength20140102Night"
    inputFileHandler = open(inputFileName,"r")
    for line in inputFileHandler.readlines():
        lineElements = line.strip().split("\t")
        currentClassLabel = lineElements[0]
        currentClassLowerBound = float(lineElements[1])
        if currentClassLowerBound < 50:
            outputFileHandler.write(currentClassLabel + " " + str( int(currentClassLowerBound) ) + " " + "1" + "\n")
        else:
            # this currentClassLowerBound can be subdivided
            outputLine = currentClassLabel + " " + str( int(currentClassLowerBound) ) + " ";
            restOfTheNum = currentClassLowerBound
            print "-->",restOfTheNum
            pieceID = 0
            while restOfTheNum >= 50:
                numOfCurrentPiece = int(restOfTheNum * stepFactor)
                print "-->",pieceID,numOfCurrentPiece
                pieceID += 1
                restOfTheNum = restOfTheNum - numOfCurrentPiece
            print "-->",pieceID,restOfTheNum
            print 
            outputLine = outputLine + str(pieceID + 1) + "\n";
            outputFileHandler.write(outputLine)
    inputFileHandler.close()
    outputFileHandler.close()
    print "Overall:"
    print "inputFileName: ",inputFileName
    print "outputFileName: ",outputFileName
    exit(1)

def step0():
    # the threshold being 50
    # original step factor
    # stepFactor = float(1.0 / 2.0)
    # modified step factor
    stepFactor = float(1.0 / 3.0)
    
    # outputFileName = "/home/diaosi/gov2ClearYourMindAndDoItAgain/relRankFilesRelated/termPieceInfoForRandomPostingTerms_stepGap_one1rd"
    outputFileName = "/home/diaosi/gov2ClearYourMindAndDoItAgain/relRankFilesRelated/termPieceInfoForQueryTerms_stepGap_one3rd"
    outputFileHandler = open(outputFileName,"w")
    
    # the threshold being 50
    classLabelWithNumOfPiecesDict = {}
    inputAuxFileName = "/home/diaosi/gov2ClearYourMindAndDoItAgain/relRankFilesRelated/predefinedClassLabelLowerBoundBasedOnListLength_numOfPieces_stepGap_one3rd_20140118Night"
    inputFileHanlder = open(inputAuxFileName,"r")
    for line in inputFileHanlder.readlines():
        lineElements = line.strip().split(" ")
        classLabel = lineElements[0]
        numOfPiecesDecided = int(lineElements[2])
        if classLabel not in classLabelWithNumOfPiecesDict:
             classLabelWithNumOfPiecesDict[classLabel] = numOfPiecesDecided
    inputFileHanlder.close()
    print "classLabelWithNumOfPiecesDict['26']: ",classLabelWithNumOfPiecesDict['26']
        
    # inputFileName1 = "/home/diaosi/gov2ClearYourMindAndDoItAgain/relRankFilesRelated/termPieceInfoForRandomPostingTerms_stepGap_2_OLD"
    inputFileName1 = "/home/diaosi/gov2ClearYourMindAndDoItAgain/relRankFilesRelated/termPieceInfoForQueryTerms_stepGap_2_OLD"
    inputFileHandler = open(inputFileName1,"r")
    for line in inputFileHandler.readlines():
        lineElements = line.strip().split(" ")
        currentTerm = lineElements[0]
        lengthOfTheListForCurrentTerm = int(lineElements[1])
        classLabel = lineElements[2]
        numOfPiecesDecided = classLabelWithNumOfPiecesDict[classLabel]
        print currentTerm,lengthOfTheListForCurrentTerm,classLabel,numOfPiecesDecided
        outputLine = currentTerm + " " + str(lengthOfTheListForCurrentTerm) + " " + classLabel + " " + str(numOfPiecesDecided) + " " 
        
        restOfTheNum = lengthOfTheListForCurrentTerm
        # debug
        # print "-->",restOfTheNum
        pieceID = 0
        while pieceID <= numOfPiecesDecided - 2:
            numOfCurrentPiece = int(restOfTheNum * stepFactor)
            # debug
            # print "-->",pieceID,numOfCurrentPiece
            outputLine = outputLine + str(pieceID) + " " + str(numOfCurrentPiece) + " "
            pieceID += 1
            restOfTheNum = restOfTheNum - numOfCurrentPiece
        
        # debug
        # print "-->",pieceID,restOfTheNum
        # print
        outputLine = outputLine + str(pieceID) + " " + str(restOfTheNum) + "\n"
        outputFileHandler.write(outputLine)
    inputFileHandler.close()
    outputFileHandler.close()
    print "Overall:"
    print "inputAuxFileName: ",inputAuxFileName
    print "inputFileName1: ",inputFileName1
    print "outputFileName: ",outputFileName


# actual construction of the denominator table

def step1():
    # I need 2 things in order to build the denominator table
    # (1) the freq of the term in the training query log
    # (2) the # of postings in each range
    # key: actual term for training
    # value: real freq of this term appeared in the traing query log
    trainingQueryTermsWithFreqDict = {}
    # for gov2 dataset
    # on dodo:
    # inputFileName0 = "/home/diaosi/gov2ClearYourMindAndDoItAgain/relRankFilesRelated/realFreqOfTermsIn_100KQueries_0_1_95%"
    # on notebook:
    # inputFileName0 = "/Users/weijiang/Desktop/dataWorkOn20140111Night/realFreqOfTermsIn_100KQueries_0_1_95%"
    # on moa:
    # inputFileName0 = "/home/diaosi/workspace/web-search-engine-wei-2014-April/data/realFreqOfTermsIn_100KQueries_0_1_95%"
    # for clueweb09B dataset
    # on vidaserver1
    # for gov2
    inputFileName0 = "/home/vgc/wei/workspace/NYU_IRTK/data/firstFactorProbability/realFreqOfTermsIn_100KQueries_0_1_95%"
    # for clueweb09B
    # inputFileName0 = "/local_scratch/wei/workspace/NYU_IRTK/data/LMs/Clueweb09B/firstFactorProbablity/realFreqOfTermsFor_clueweb09B_queries_ALL_sortedByRandom_20140609_training_sortedByTermFreq"
    inputFileHandler = open(inputFileName0,"r")
    
    for index,line in enumerate( inputFileHandler.readlines() ):
        lineElements = line.strip().split(" ")
        currentTerm = lineElements[0]
        currentTermRealFreqInTrainingQueires = int(lineElements[1])
        # ignore the lineElements[1] cause I don't need this info
        if currentTerm not in trainingQueryTermsWithFreqDict:
            trainingQueryTermsWithFreqDict[currentTerm] = currentTermRealFreqInTrainingQueires
        else:
            print "duplicated terms found."
            exit(1)
    print "len(trainingQueryTermsWithFreqDict):",len(trainingQueryTermsWithFreqDict)
    # print "trainingQueryTermsWithFreqDict['of']:",trainingQueryTermsWithFreqDict['of']
    # print "trainingQueryTermsWithFreqDict['fawn']:",trainingQueryTermsWithFreqDict['fawn']
    inputFileHandler.close()
    
    twoDimensionDenominatorTable = {}
    # for gov2
    # option1:
    # inputFileName1 = "/home/diaosi/gov2ClearYourMindAndDoItAgain/relRankFilesRelated/termPieceInfoForQueryTerms_stepGap_one3rd"
    # option2:
    # inputFileName1 = "/Users/weijiang/Desktop/dataWorkOn20140111Night/termPieceInfoForQueryTerms_For_100KQueries"
    # option3:
    # inputFileName1 = "/home/diaosi/workspace/web-search-engine-wei-2014-April/data/secondFactorProbability/termPieceInfoForQueryTerms_stepGap_2_OLD"
    # on vidaserver1
    # for gov2
    inputFileName1 = "/home/vgc/wei/workspace/NYU_IRTK/data/secondFactorProbability/gov2/termPieceInfoForQueryTerms_stepGap_2_OLD"
    # for clueweb09B
    # inputFileName1 = "/local_scratch/wei/workspace/NYU_IRTK/data/secondFactorProbability/termPieceInfo_clueweb09B_20140613"
    inputFileHandler = open(inputFileName1,"r")
    for index,line in enumerate( inputFileHandler.readlines() ):
        # if index == 3:
        #    break
        lineElements = line.strip().split(" ")
        # gov2
        base = 0
        # clueweb09B
        # base = 1
        currentTerm = lineElements[base + 0]
        currentClassBelongingTo = int( lineElements[base + 2] )
        numOfRanges = int( lineElements[base + 3] )
        if currentTerm in trainingQueryTermsWithFreqDict:
            # for debug
            print currentTerm,currentClassBelongingTo,numOfRanges
            for index in range((base + 4),len(lineElements),2):
                rangeID = int(lineElements[index])
                numOfPostingInThatRange = int(lineElements[index+1])
                print rangeID,numOfPostingInThatRange
    
                if currentClassBelongingTo not in twoDimensionDenominatorTable:
                    twoDimensionDenominatorTable[currentClassBelongingTo] = {}
    
                if rangeID not in twoDimensionDenominatorTable[currentClassBelongingTo]:
                    twoDimensionDenominatorTable[currentClassBelongingTo][rangeID] = 0
    
                twoDimensionDenominatorTable[currentClassBelongingTo][rangeID] += trainingQueryTermsWithFreqDict[currentTerm] * numOfPostingInThatRange
        else:
            pass # NO NEED to do sth
    
    # print "twoDimensionDenominatorTable[58]: ",twoDimensionDenominatorTable[58]
    # print "twoDimensionDenominatorTable[63]: ",twoDimensionDenominatorTable[63]
    tempCounter = 0
    for i in range(0,len(twoDimensionDenominatorTable)):
        print i,
        for j in range(0,len(twoDimensionDenominatorTable[i])):
            print twoDimensionDenominatorTable[i][j],
            tempCounter += twoDimensionDenominatorTable[i][j]
        print
    inputFileHandler.close()
    print "Overall Processing Statistics:"
    print "inputFileName: ",inputFileName0
    print "inputFileName1: ",inputFileName1
    print "tempCounter: ",tempCounter

# function for building the denominator table
step1()
print "Program Ends."








